{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>模型优化，增加隐层和神经元，修改初始化方式，增加正则等\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入tensorflow和mnist数据集包\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-c7061c95ced6>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/yaoruiqi/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/yaoruiqi/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./mnist_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/yaoruiqi/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./mnist_data/train-labels-idx1-ubyte.gz\n",
      "Extracting ./mnist_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./mnist_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/yaoruiqi/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "mnist=input_data.read_data_sets('./mnist_data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(mnist.train.labels[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55000,)\n",
      "(55000, 784)\n",
      "(5000,)\n",
      "(5000, 784)\n"
     ]
    }
   ],
   "source": [
    "print(mnist.train.labels.shape)\n",
    "print(mnist.train.images.shape)\n",
    "print(mnist.validation.labels.shape)\n",
    "print(mnist.validation.images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr8AAAK7CAYAAAD7iOf4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XeclOX1//9zbaFXQVG6lAXBglHsRhQrYq+xG5Uo1miMRmOM8ZOYmBgbNoTYuwYVG1bsCmJBkF6UKtL7suX6/rHr72d4X8i9s7M7s3O9no+HD+Ob+5450XuHMzdz5jjvvQEAAAAxyMt0AQAAAEBtofkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA2a31rinPPOuTXOub8mPP4c59zqyvO61XR9wKZw7aKuSuHavaHyeO+cK6jp+oAQXnNrnmPJRe1wznkz6+69n175z/ua2asbHdbYzI733j+3qfOA2ha4dlub2Qtm1tPM8s1skpn9znv/4c+dB9S2n7sGnXNnmtmDZnae937YT/LOZjbLzAq996W1Uynw/wtdt865fDO7wcx+bWZNzWy6me3vvV/+c+chjHe2GeK9f9/Mmvz4z865fmY20sxey1RNQEKrreIFeJqZeTM7ysxGOue2ollAXeCca2lmfzCziZmuBUjoBjPby8z2NLPvzKy3ma3PaEV1GB97yB5nmtmz3vs1mS4E+Dne+/Xe+yne+3Izc2ZWZmYtzWyLzFYGJHaTmd1hZoszXQiwOZVv1i6zij+l+NZXmOC9p/lNEc1vFnDONTKz483soUzXAiTlnBtvFXceXjSzYd77RRkuCdgs59xuZrarmd2b6VqAhHYws1IzO945t9A5N9U5d2Gmi6rL+NhDdjjOKu5AvJvpQoCkvPc7OucamNkxZlYv0/UAm1P5ucm7zexi7325cy7TJQFJtDez5mZWZGbbmll3M3vLOTfVe/9GRiuro7jzmx3ONLOHPdOHqGMqPwLxhJld7ZzbKdP1AJsx2MzGe+8/znQhQBWsq/z7X7z367z3483sSTMbkMGa6jSa3wxzznUws35m9nCGSwGqo9DMumS6CGAz+pvZMZV/dLzQKgaIbnHODclwXcDPGV/5d26QpQkfe8i8083sI+/9jEwXAiThnNvDKl47xljFV51dYmZtzOzTTNYFJHCWmTX4yT//18yeNbPhGakGSMB7P8M5976ZXeucu8QqbjScZGa/ymxldRfNb+adYWb/zHQRQBXUt4pJ+S5mVmJmX5vZ4d77+RmtCtiMn34nqpmZc26Dma303q/IUElAUr+yijdpS8xskZld571/K7Ml1V187KH2FJvZOOfcjT8Nvfc9vfdy18E5d7ZzbnnleeW1VCMQ8j/Xrvf+Xe/9Tt77pt77Lbz3+3nv3/vxYK5dZJHg6+6PvPf9Nlpwcb2ZfVV5Hn/EjEyR69Z7P897f6j3von3vov3/r4ff43X3KpjwxsAAACiwZ1fAAAARIPmFwAAANGo1YG3g/JO4DMWqLI3yp/J+DfRc+0iFZm+drlukYpMX7dmXLtITdJrlzu/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKJRkOkCAAAAUA15+RJNvX9nySYecrdkR5x5gWQFb41LT11Ziju/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGgy8AQAA1BEFnTpINvWmVpLN6jcscHY9SZZ31az1WymVVmdw5xcAAADRoPkFAABANGh+AQAAEA2aXwAAAESDgTcAieX3KpJs8gUtJZt27D2SlZuXLM+cZHcv31ayh/49QLJWwz/eZJ0AkAsKunSW7JtrW0sWHm5T583ZW7I27y+WrCzRo9Vd3PkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRYOANgBV0aC/ZN9dvLdkTB9wn2c71yyUrD7yvLjc9LvT+e1CL6ZK1veoxyf4zal/JSufOCzwHckFegwaSdXwvMDDZ7kPJ8p1eZ5M2rJXsikPOkKxsil6PQE1whbppbdKft5Bs1oHJhtu6vPlryXoM+kay8vXTEj1eLuHOLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAYDb2m04PK9JHO61MoaLNFwWU89bpuPdcdKg5FjUqoN+NHMm/eUbPKpd0mWdCNbaLjt5bXNJRuzukui+nZpPFuy45qslGz+qAmSvdRbt82h7gkNt817Ujf/vdROByFD+k04WjJ3i27Jqj/jy0SPVx0FnTtKVjr7uxp/XmS/KUN2kmzWgfcnOrfb6LMk637G55KFxo5jxJ1fAAAARIPmFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARKPOfNvDogv1mxSW71gi2YiDh9RGOUHb1Rub6Lj1vlSy5nkNJVt0+hrJ5t+h/8n+vfAgyZac2Eyy0jlzE9WH3HbCQbr+NfTNDknXEd+1vKtkbxzSW7Kkq4c/POJkyY689x7JQmuQX7K+iZ4D2W36n3eWbHJf/UaSkO5vnStZjwumSFa+ZrZkgS/nqZapQ/V6fOHgOyU76cHLg+d3/PNHaa4I2WL6bXtoNvDuwJH6mtvlDV1bXDRoomTpvp5zCXd+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANLJy4G3q/TokMHnA7ZLVd4WBs+vXQEXpFa5bbZXfOJDpcQ93ek+y057qJ9myU1irGZ3ddpDo/FY6PPby2q0lC60jnrCyrWTFV24p2Yyb9UIturGRZGWTpkkWWuFdeJ8+XklgmmPeVToYa2bW7h8MDmUrv6eudH3vlH8GjtTr57vStZIVnaNrr8tLNqRUW1WUHLiLZCMO0gHs3oX1arwWZJcNh2pPM+Lo2yTLd7rWO7i2+OyvJPPlZakVFynu/AIAACAaNL8AAACIBs0vAAAAokHzCwAAgGhk5cDbPfs/LFloSOwfS7pLtmhD07TX899xOsjQcaRL+/NsbG5/fW9y84DHJTuuyUrJHu08WrLTHu8n2bKT2kvGJrgcMuZriQYdd4Fk+QuWShbeyLZQknlX6WDcpP10i9Vh95+nzztJn2HJOXtKVuLHSRbaQNfpsW/1Ac1MdyoiW3x/lQ6jbZWvw23rvB53xmVXSNao5NP0FFZFq3+rr8M71NPft1b7Ysm2fWZJ8DEZYcoNra6dJdmO9XS47aBJR0hWdL1eV2UMt1Ubd34BAAAQDZpfAAAARIPmFwAAANGg+QUAAEA0snLg7baTjpfsj32aSbbV81MkK1uigzvVVWRj0/6YSXQbqdmw/wyQbOGTH0p2YYs5koWG4HoM0uGnztcx8JbL/FgdgqvOQFiDxbpqbeiKzpLV+361ZDNv0I1sD56uw3J5pgOm44r1vXt4SA/ZbFDRB4mOO2bKCZI1GpFsuM0V6G91rmHDROeGlO2gQ563bvdAonP7jTtbsq0mTk65FmS/S9q9mei4lQ/pAHqLaR+nuxwYd34BAAAQEZpfAAAARIPmFwAAANGg+QUAAEA0snLgzY+bKFkrXfAU5fab8vE6GPHArQMlu/CGexI93uOn3S7ZNdftVvXCUKetO0r/my/tqS8PoeG2Vl/rINug5rMl6/OSbl/brb4+Xmhz29jAcNsfzwlsjLPPJUNuaFq4XrI1geNKDt5Vsi2umy3ZU11er0Y17yY66sPAdbvl3+tX43mR7Vactodkv2zwpWR7jz9WshaPfFIjNUFx5xcAAADRoPkFAABANGh+AQAAEA2aXwAAAEQjKwfeANSu+SdtkGzSfjo0Gdq0Vm46tBY6LjTclnRz2+nPXiRZl3fYfJQLht5/hGTn/26IZA930ZWX5390qGTDO+l1W2D5KVZXPWeNPF+y7h8z1JTLlh8ZGsNUa1/cWrImfma6y0ldXuBnpjx3vmaAO78AAACIBs0vAAAAokHzCwAAgGjQ/AIAACAaDLzVMXOv2Uuy8p1Xpfx4bfJ10Kn0gF2Cxxa8HVizh5wV2rQWer9cneMGzTlAsjl/6C4Zw225a0370PWjGrp6kj3U6e3AkTqoc8VC3WD4yqi+kpVso6+H0w++P1F9Ia0/14FO5LZtWq5MdFzDJcmu+3QrPkyv+8XnrZVs+zYLJFt1vP4Mli5YmJ7Cahl3fgEAABANml8AAABEg+YXAAAA0aD5BQAAQDQYeEujgi6dJZt+zjaS3X3y0JSfo1+DzyXLd6m/h2lf0ESyoQ/cHjx2cKd9Un4eZLe2T+kgwwntdPPW9s3mS3Z+q48ka5ffKPAsep3OuGk7yRq+M2YTVSIXFd33g2TblVyY8uN1e2SpZOVTZki2bakOUc78+54pP+/geXtLtsXjOiSsew5RVxVs3Uay+3s8FjhSf59Nt/wWzSU7+uNpkp3U9A7Jmuc1TPQcvYecKln74xh4AwAAALIazS8AAACiQfMLAACAaND8AgAAIBoMvCWw+oTdJfvhF/q+4S/HPinZyU2Xpbmamn+/cuCblwXzIvusxp8bmdHwBR0yK35BjxsXuP4G9b1AslU3rpHs7R2ekmyfP38i2VfjOkhWOneeFoOcUDY1MIx2tWaJH68atRSsTX0j22fD+kjWuoTNhDmtsFCijoEh8nRbNFg3vR79m9GSDWquA8pmyYbbQrZsqq/rdRV3fgEAABANml8AAABEg+YXAAAA0aD5BQAAQDSiHXhzO/cO5i2GLJDslc73SFadrWrPr9EPxE9Y1z7RuS/d3E9rKdadQWf+ZaRk4Q+/q3oL9UP8yIyCDnpdlM6Zm4FKNs2P/VqyJofqcSe8qxvjRnR7RbLtz9VNgh3/zMAbap5LOC1XGhirazm1OM3VINv5VaskG7qirWRJf+/Nb91Ksjm/7iHZ15fdnejx0m3FugaSbZWBOtKBO78AAACIBs0vAAAAokHzCwAAgGjQ/AIAACAaUQy8fXuDbkO57mTdNmVmdmrTJZJ9V7pWsskbWkp28RPnStZogW4M2mb0YsnKvpkarGdjzU03YoVM+0ObwMn6oftZJasl6/yCZqh5647aTbLQBrSXvg0Pa25z9KS015ROK/7VUbLye3VYs6T7utooBxBn/2pUouNOmK7Dm/mjP093OchyZctXSPbE3L6SDWqu6zL3vupTyfreOFOyE5u8lWJ11XPDD70ka3uJbngrrY1iagB3fgEAABANml8AAABEg+YXAAAA0aD5BQAAQDSiGHhr0XeRZKHBNjOz/t8cKVnJnVtL1vCFMZJ1to8T1ZNwiVBi5fvtLNnRLYYHjtT3OkvL6+lhY3RjF9IrtLntpJteleyzlZ0ly/bBNjOz/BbNJTv+7zpMlGc6EArUhvwtt5Sse/3pic5dfE9nyZrawuqWhByw/oFtJCv+Z4lk/9z6i9ooR5R47UB6vXuOZEV/0B6p9Ns5NVJTJnDnFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARCOKgbdW5+jGsm6XXxA8tuuVOrRWYN+lvaZ0WlbUQLK9GyR7XzNowmmStbZk2+aQum9P0W1noS1At35xoGRdLTODEpu02w4SHfbAe5INaqHDROWB99+FUxumpy7gZ6zYv6tkRzTSoczVvliyBot1gAkwM2v2uG7l/PT/CiX7pf62XS1lvlyyXT87RbJ6z+p22i6PaN9TVze3JcWdXwAAAESD5hcAAADRoPkFAABANGh+AQAAEI0oBt5KF+jmna5X5s42niV9k300fdKGtZI1vVs3caHmtXtnlWSFl+ZLdmmftyUbfvHhwcdsNVEHcwreHpeonvxeRZLN799asiaH68/NOzs8KFloc1touK3o1d9odsNHmyoTSJszb3gx0XGzSgJDmW8m+7kCqqLnB6dL5iY0lWzbOyZK5st04G2rVZPTU1gO4s4vAAAAokHzCwAAgGjQ/AIAACAaNL8AAACIBs0vAAAAohHFtz3kkkMmrJRsRIu7AkfWk+TMiWdK1vLVsekoC1U15muJ9h5/rGRv7/CUZOdffWfwIctNp31vWLRLonKObP6EZDvX18fLC7xfDj1v6H11j2cvlKzXP+dIlutrNZEdWuXr2vuQfy04JJAuT28xiE6vewZL1vmmMZL5Un1FLKuRiuLCnV8AAABEg+YXAAAA0aD5BQAAQDRofgEAABANBt7qmOObjZesUV4TyaaWrNHjhrSokZqQHi3O2yDZDS/qwNrf2ug1YGZW4jW7casvJSs3PTDpOuLvy9ZJdveSvSR7fcjeknUf/rFkDLch220o17XjQFX8tUsfyTqYrnEPvISjhnDnFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARIOBtyy2aLAOErXJ141ss0p0U9Gv/nalZK1f1YEjZI/SOXMl++qIDpJ1+0eyrW1mZpP6DZPsl+NPlOyHpc0SPV6323REzY/VbXWtjGsNueH+zi9Jtsstv5Ws6xWf1EY5ANKAO78AAACIBs0vAAAAokHzCwAAgGjQ/AIAACAaDLxlCVe/vmTHnf+2ZKvKdQvYgDEXSNbxPgaOckHp3HmSdT1Vs00ZaDoc18xmBLJk2ECEXHHtk6dK1vOMf2tWqK/NVq4bEQHUHdz5BQAAQDRofgEAABANml8AAABEg+YXAAAA0WDgLVuU6yjRIyP3l+zVr/pJ1vFpNgsBQFV0+pMOBV/+pz0TnduVDYZAncadXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA0G3rKEL9HNbZ2vZagCAAAgnbjzCwAAgGjQ/AIAACAaNL8AAACIBs0vAAAAouG8181iAAAAQC7izi8AAACiQfMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiQfNbS5xz3jm3xjn314THn+OcW115Xrearg/YlBSu3QMrr91y59yBNV0fsCm87qIuSuG6vaHyeO+cK6jp+nIBzW/t2sl7f+2P/+CcG+qcm1LZJJz10wO998O9901qvUIgbONr9wDn3OfOuZXOuZnOuUE//pr3/s3Ka/e7jFQK/C9ed1EXbXzd9nHOjXPOra38e58ff817f72Z9c5IlXUUzW9mfWVmg83s80wXAiTlnCs0sxFmdp+ZNTezk8zs3865nTJaGJAMr7uoU5xz9czsBTN71MxamtlDZvZCZY4U0PxmkPf+Lu/9W2a2PtO1AFWwhZk1M7NHfIWxZjbJzHpltixg83jdRR3Uz8wKzOw2732x9/4OM3NmdkBGq6rDaH4BVIn3/nsze8LMznbO5Tvn9jSzTmb2QWYrA4Cc1NvMxnvv/U+y8cZHHVLGB6MBpOIJMxtmZrdX/vMF3vs5GawHAHJVEzNbsVG2wsyaZqCWnMCdXwBV4pzraWZPmdkZZlbPKu4+/N45d3hGCwOA3LTaKj5q9lPNzGxVBmrJCTS/AKpqezOb4r0f5b0v995PMbOXzeywDNcFALloopnt6JxzP8l2rMyRAprfDHLO1XPONbCKD64XOucaOOf4b4Js94WZda/8ujPnnOtqZgOtYooeyGq87qIOGm1mZWZ2iXOuvnPuosr87cyVVLfxA59Zr5vZOjPby8yGVv7vX2a0ImAzvPczzOzXZnaHma00s3fN7DkzG57JuoCEeN1FneK932BmR1vFR82WW8Xr79GVOVJA81t7is1snHPuxh8D730/773b6K/RZmbOubOdc8srzyvPTMmAmYWv3ae999t775t679t776/y3pebmTnn+ldeu22s4m4FkCm87qIuCl23X3jvd/HeN/Te/8J7/8WPv+acu94q/uSt2My8Phw25v73mzMAAACA3MWdXwAAAESD5hcAAADRoPkFAABANGp1w9tBeSfwAWNU2Rvlz7jNH1WzuHaRikxfu1y3SEWmr1szrl2kJum1y51fAAAARIPmFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARIPmFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARIPmFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARIPmFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARIPmFwAAANEoyHQBqDDryR0l+2DveyQ75YyLJct/5/MaqQkAYjLjlj0ku/TQVyV75Vd7SlY+fnKN1ITI7aG9waxLnWRT93tIsm6jz5Ks6ylfpqWsuo47vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBoMvGUJ/11jyVrt21CypT3qS7blOzVSEpCy4sP7Srb0vNWSfdH3sZSf4/y5+0r2was7BY/tct9MyUoXLEz5uVH3FbRrK9mQox6Q7KCG6yR7aPcBkrUan566EK+Fl+0l2d8u+o9kBzdcI1mJ18e7fbcnJbvDeiaq5fuLtZa2j+tQZ9mSpYkeL9tw5xcAAADRoPkFAABANGh+AQAAEA2aXwAAAESDgbcs0XiubmwJ2fqkbyUruzfd1QBhrrCeZFP/vbNkLx9xq2TdCnVYs7watdzb/n19vPPeCx7bZ4czJGt/HANvMZvxm06ShYbbgOpy9fW1b9mJv5Dsvd/dIlkjp6+56Tb3DzrcNvbC2yR7+sL2kt1x23HBx9zy3o+rX1gN4s4vAAAAokHzCwAAgGjQ/AIAACAaNL8AAACIBgNvdcy60kLJav7j8ECFKXf2kWzqEXdLlmcNJCu3wAqihAbN6SfZsA7vJj7/jj666eiWVvtJVle3FaHqOuw9N9MlIBIz/6zDbRPPGBI4MvXfze9d3kWy+x45XLJ29pFkxa109LjQ5Ut2atMFkvW9+t/Bek63yyXLpiE47vwCAAAgGjS/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGnzbQ5ZodrhOUYaseK6tZFuarjwGqiK0tjj0zQ4TB4YmlHUqeEHZWsl+OeJ3knUZsUGy+tN07XDZ4iWS7fzUqZKN6/tooD6zz9d1lsxvKAkei9yzfuBukt3e5c7AkfptOkBVhFYZN+61LK3P8erappI99/uDJWv3sn6zQ7oVBX7vMDN78g//kuyQnS/T838zNu01JcGdXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA0G3jKgrJ+uOhzZ+y7Jvtygg0RtHpsgmS4mBKpmwYW7Sjb1iNBAkF6Tw1d0lOy/5x0kWfcPP0lUS2mio8yKi5MPJ42ct6NkDVfNSnw+6rZ1rfS63aEew22oHlegLdSMv+jv79/sGhoUTia02n3RcTrwVn9e6oNjnV/WweMdO50l2bg9h0sWWoNsZrZtga64bzY5e37muPMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiwcBbBpTV1/ccTZxuhSnxXrLyVatqpCbE7YJBL0iWZ06ym5b0kuzjI4skc7O/TLmW/GbNJJt77vaS/X7H/0r2xYbw+GfDQxhuQ2o+LNbX66Zzko5lIpcVH7izZN+clvpw26Xz95bs+8N1SKxsyfyUnyMk/53PJev4jh43Yso2kp3YZFFaa6kt3PkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRYOAtA2Yfw3sOZJeywPvgctOBy1f+1k+yprOTbW6zPN0EVLbfTpINHPKWZOe30OmL0EDe4VOO3sSTz9t8fchZPc+fmPK5t83VbYX1Xkt9mxbqnu8v2SuYD77g+ZQfMzTcNmu/wOvw2qUpPwc2jS4MAAAA0aD5BQAAQDRofgEAABANml8AAABEg4G3DGi6NVvaUDc1Wrgh5XNDw22vPnp/yo93zPQBkuUdtzb83Ck/C3LB4DaBdVWBgcmQKa92l6y9/VDNipCt8nbaTrK/XzI8eGz/huHXm40NmtNPstDmtmwabnM795asc6FugtuU6SXFkjWfmT2bEbnzCwAAgGjQ/AIAACAaNL8AAACIBs0vAAAAosHAGwCbtq6Nhs1nS/Sfh++Q7O/fHyjZ6G+7SfbabnquWUNJVpSvl6zvy7+VrOcVurWrfM2awHMAqev0vA63MUCZu/Z9RIe6kg62bcrY53eQrN2Sj6r1mDVtygWNJNutvm793JRRa3pJ1vCFMdWqKZ248wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKLBwFsNy2vQQLJ92s1KdO79i/YLpKurWRGgJl2owwn23KcSbZOvA2q3t/1Qsry2OsxRHhhuC9n/zislK7o59HiA+v6SvSTrURgaLtLX5nllgcGmUsbbctXi3+wp2QUtbwkcWT94/oKydZJd/u3RknX87/eSZdNVVbBtJ8nePfTWwJHJXsPNzD5YqkPPZouTF1XDuPMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiwcBbDctr0VyyO9u+mujcdz/YXrKu9km1a0Lcig/vK9mck0slyzOX8nPku8D7aq8jav0nHitZ28BwGxCS32YryXY+5WvJmuXpcFtIvxG/k6z7NF5zc9UqnfOyJnnh4baQfy3aXx9z39BQV/YMeoVMuXAbyULDzSHLAhs5zcwW3t5VssZZ9O+BO78AAACIBs0vAAAAokHzCwAAgGjQ/AIAACAaDLzVsNLObVI+t+NrJWmsBLkub8eekm09dJ5kwzrcJ1m5+UCWzNULdYDuv2N2leyegx6SbHiPRyU740QdOmryNENHCGjdUqJhHV5LdOrKwKBO01ncD0Jyr72pr3Pb2scZqKQKnA4y+/zUH+53cw8L5o2f1Q2h2YSfdAAAAESD5hcAAADRoPkFAABANGh+AQAAEA0G3mrY4mvD2082NmDykZLVG/2VZDqWhNgsHrRnMB913b8kax7cbJVsc9sVC/aQ7NW3dcCj6NZZmi0YI9m/9j9VH+/R+yU7+XrdgPjS0zrYBJQ1rpfyuV+XNJJs69vYLojktvmwLNMlVNmKU3eXbPKJd6X8eB992CuYZ/s2Wu78AgAAIBo0vwAAAIgGzS8AAACiQfMLAACAaDDwVsPu2f6xQKrrVOavbCZZ29K5NVAR6pJVJ+vQWWiwzSw83DapRLcE3rrwIMmm3NZbH+/5LyXrsl63F5UGq1H57+oAZ8+nL5TsqxNuk2zEwRdJVvj6ZwmfGbnfViM8AAAgAElEQVSq6S0LUj73gi90ALO9TaxOOYhMp2smS/b9yAwUYmYF7dtJNu3CjpJ9etotgbPrJ3qOJ1bpxtqiB5YFj832UUDu/AIAACAaNL8AAACIBs0vAAAAokHzCwAAgGgw8JZGBZ31w+VNnW4MyneFtVEOcsDiHXUbW3hrm9mINVtI9sCJh0tW/uU3kjUNbOMpT1JgFeQ11Lp7/2K2ZPUDPx/lBcm20iF3FXRoL1lRk+8SnXvq7AMl63TufMmyfUgH2WWfFtMle767DimXTZuZ8nPkb9ddsmlntpbstuMfkOzghmsCj5hsuC3koQuPkqxg4riUHy+TuPMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiwcBbGq0fpllRoQ75lHkdJWrytG54A0LyLDz8ddU7J0pW9OXYmi4nKL91K8kajdC6n+rySuBshtugFg7oINmLW70oWb7TezrL1jeSLG+DbqZyhfUk8yUbkpaIOqb7MN0QeMOAPpJdv6VuuzQzO7vZHMnyX9Tf379eq8OaSfVp/K5kpzZNfbNhyItrWkr2uzdPlqznJ7oBMd2D0bWFO78AAACIBs0vAAAAokHzCwAAgGjQ/AIAACAaDLylKL+oq2RXdNbhi5BfzTpIsmZPflrtmpB7Wo/3ki0rXxc8duyA2yTre99lkm33p28lK/t+UaJ6Ctq1lWzNTu0ku+z2JyQ7vNEKyULDEnct15+thu9PTnQuEBoofqVn4LV5qkbdnx2s2aW6/RC5oXTmbMlG3bGPZJfdEL4GQts2z2g2Tw8MZbVgrddhzbuW6kDfe7/uK1nRZ2Mky6XXXO78AgAAIBo0vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBp820OKNrRrLln/hsWJzp36VA/J2viPql0Tck/TJ3XK+Jfdrgwe+9UFd0o2deC9kk08uFSyy6adlKiex7Z7TLLQxHNoBXNoUviKBXtINvniXpK5VV8lqg+5q8FSvYJmlOo3n3QtaJjo8dYFJuEbLeB+UOy2+M/Hkv3pgv7BY8/fcrRk2xUWprukRELfkvPI7YdJ1nqo/v8zm1ADFWU3ftIBAAAQDZpfAAAARIPmFwAAANGg+QUAAEA0GHirYefP3Veytk9MkaysNopBTthicvhquXd5F8l6NZgrWb8GOoz2Ru/nEj67DrcFa1nRSbJbXx4oWffrvpDMrWe4DarJM7oC/sStdfjzyz/cLdn/Le4p2XNDD5Cs3RAGj6Fm9F0fzK/u9is99qytJTvk0M8ku2UbHWbu/fBFkrmEzUHXx5dI1vqb0HAbzLjzCwAAgIjQ/AIAACAaNL8AAACIBs0vAAAAouG897X2ZAflnVB7T4ac8Ub5MzqhVcvq6rVb0LmjZNP+3iLRuTf94nnJPlrVTbKRo3aXbNtrGLQwy/y1W1evW2RWpq9bM65dpCbptcudXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA02vAE5rHT2d5Jte7JmIUNNN8aZlevjGcNtAIC6gzu/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKLhvPeZrgEAAACoFdz5BQAAQDRofgEAABANml8AAABEg+YXAAAA0aD5BQAAQDRofgEAABANml8AAABEg+YXAAAA0aD5BQAAQDRofgEAABANml8AAABEg+YXAAAA0aD5BQAAQDRofgEAABANmt9a4pzzzrk1zrm/Jjz+HOfc6srzutV0fcCmpHDtHlh57ZY75w6s6fqATUnh2r2h8njvnCuo6fqAEPqFmkfzW7t28t5f++M/OOeOcM5NqLxoP3LO9frx17z3w733TTJTJiA2vnYPcM597pxb6Zyb6Zwb9OOvee/frLx2v8tIpcD/2vja7eOcG+ecW1v59z4//pr3/noz652RKoH/9f9dt8651s65D51zS5xzy51zHzvn9v7xQPqFqqP5zRDnXHcze8zMzjezFmY20sxe5G4Dsp1zrtDMRpjZfWbW3MxOMrN/O+d2ymhhwGY45+qZ2Qtm9qiZtTSzh8zshcocyFarzezXZralVVy3/zCzkfQLqaP5zZxDzOx97/0H3vtSq7iY25nZfpktC9isLcysmZk94iuMNbNJZtbr508DMq6fmRWY2W3e+2Lv/R1m5szsgIxWBfwM7/167/0U7325VVyvZVbRBG+R2crqLprfzHGVf238z9tnphwgGe/992b2hJmd7ZzLd87taWadzOyDzFYGbFZvMxvvvfc/ycYbH3VAHeCcG29m683sRTMb5r1flOGS6iya38x5w8z2c871q/wjt2vMrJ6ZNcpsWUAiT5jZn8ys2MzeN7NrvfdzMlsSsFlNzGzFRtkKM2uagVqAKvHe72gVf+p2inGzoVpofjPEez/ZzM40syFmtsDMWpvZN2Y2N5N1AZvjnOtpZk+Z2RlW8Yatt5n93jl3eEYLAzZvtVU0Dz/VzMxWZaAWoMoqPwLxhJldzZxF6mh+M8h7/6z3fnvvfSszu94q/uh4bIbLAjZnezOb4r0f5b0v995PMbOXzeywDNcFbM5EM9vROffTj5ztWJkDdUmhmXXJdBF1Fc1vBjnndqn8zOSWVjE5P7LyjjCQzb4ws+6VX3fmnHNdzWygmX2V4bqAzRltFcNClzjn6jvnLqrM385cScDPc87t4ZzbxzlXzznX0Dl3lZm1MbNPM11bXUXzm1m3m9lyM5tS+ffzMlsOsHne+xlW8bU7d5jZSjN718yeM7PhmawL2Bzv/QYzO9oqPrKz3Cqu46MrcyBb1Tezu8xsiZnNM7MBZna4935+Rquqw9z/Dr2ipjjn1lvFcNAd3vvrEhx/tpndamYNzKyX935mDZcIBKVw7fa3ima4vpkN8N6/U8MlAkEpXLvXm9nlVnHtNvbel9VwiYCgX6h5NL8AAACIBh97AAAAQDRofgEAABCNWt0LfVDeCXzGAlX2RvkzbvNH1SyuXaQi09cu1y1Skenr1oxrF6lJeu1y5xcAAADRoPkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANGh+AQAAEI2CTBcAIDsVdOog2fLd20m2YOAGyS74xbuSXdZyqmTbf3C2ZOWzG0vW7Yav9Li1ayXblIJttpasdMHCxOcDQKpK++8i2ZLe9SVbt5WXzHdbI9lVO70u2TnN9fXstbX6HFcOPUeytjd/JFmu484vAAAAokHzCwAAgGjQ/AIAACAaNL8AAACIBgNvAGz+lXtJdu25T0h2TJNFiR4vL/C+utzKJRu/z3A9eR+Ndlp/qWSdrk8+pFH/qTLJSn+Z+HRkA+ckWnTBnpJdcPHzkg1qPj+tpQxd0Vay54/cQ7Ly2XMl8yU6IIrcseI0vQ7e/vsdktV32n6Vmw68heSZ/iyUeD23f0MdCv7gklsk2yv/Csna35TbQ3Dc+QUAAEA0aH4BAAAQDZpfAAAARIPmFwAAANFg4K2G5e20nWRTLm8o2el9PpXs4i3GSNb/lisl2/q23P5gOtIrv1eRZNUZbvuhrFiyb0sbSVZmhZLtWk+Hf/IDg01fnXu7ZH1X6hDcNreEfxb22WKGZKOsWfBYZIG8fInmXLu7ZF+fPyTRwxV7HXicX6rXbQO99GyrfL2Wz2mmg2znjH5WstuXdZPsrYHbS1Y6+zt9YtRJK49eLVmh0+s5NNz2Xek6ya6de2Si5/10chd93sb6+vrB3vdIttfRukFzzr91O5wv1p+Zuoo7vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBoMvKXI1dcPgy8ctItkn16tgzqryvVD6Hs8+TvJ3uujwxL7nTZWsim3bbJMQEy+uolkoeG20HW6/2fnSdbm9gaS5Y/+PFEti3+jG7oGDn5PsmtafylZmf4IbtIHS7sG0h+SPwBq1bwrqzPcVirZTo/rcGSX338sWf523SWb/Iemkk044F7JQhu7Lm05XQt8SaM3+22roZmVLV4SzJG9Op83T7LBr+k6yQlLt5aspV6mVjZVh3VDimxpouN2v/e3kk09Qofg+lxxsWTt/5Y7w/Xc+QUAAEA0aH4BAAAQDZpfAAAARIPmFwAAANGg+QUAAEA0+LaHBPIa6DT75Nt2lGz6ETqNfOdynR5+5oZDJev6dGDyuEgn1Md37SOZP0J3chas1XWeBW+Nkwzx+e++Otkbeh88+Ftdq9n2mG/SWkvr+/S6f3vRPpJdM0S/7aEqprymP4ft+baHrOAK9Lehenun/i0H2/9Xp9S7B77ZIaRs0jQ99ww9bt9BOpZ/81VDJevXoESy0DdAvNV0h3BBfNtDnVO2bJlkX9yv32rTYoauCi6bmuxbcqojf02ye569B0yRbMXf0l1N5nDnFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARIOBt43kNWok2bzHO0k2va+ut/z3Mh2qGXXxfpI1eeeTRLWE1ho2WrZSsss+Hi3ZsIW6TnHFW4meFjluh3qFkpWbl2zsVF25WmQ1P4DTdIIOon2wXodOW03UNbab4nUmFFkiv2N7ycbu8kSic+9c3kWynvfqwJGO/1ZP66E6QDfivF0l69c22aAdclurYXXvOhjY+ivJHjP9Wa2ruPMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiEe3AW2iwzcxs8i3bSxYabvvX0h6SvXdkL8nyZ6V3Y8ucs3Sorn/DUZIt3VKf9+EWupWubPmK9BSGOmP/CcdJ9sb2T0v2UL9hkv3VdMNgdZT230WyLW/UQc8uBXqdtr5ilmRrXgg/j9N5PmSJ2Se1TXTcaq8bsZ78m27LbP5NsoHidJt5VmfJPhz5qWR71y+XbNqg8L+DLtfNk8yXJh/0RNyKD+sr2VkHjU507vOLdg6kubMVkzu/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGtEOvP1w6k7BfPqRd0n28tomkr13VG/JSmfNrnZdm7OhebLJnUnrdYCC4TaYmTW5TH/s73lWBykHNZ8q2dS7d5Os1z8WSPb9wboJ6IiL3pXsjBa3S9a2oL5kZpo93GWkZAMHXBw416y0IRNvmZbfaotgftWZOmwZ8uwq3TjY/LHMDLeFlE2cItmZowZJNv1IHaCedIb+vmNmdvhzZ2j42YSqF4eckt+smWTf/0p7kt9cphPA5zSbK9ns0nWSLfmn/rw1YOANAAAAqHtofgEAABANml8AAABEg+YXAAAA0Yhi4K2gnQ5//f7Kx4PHzitbK9lN1w+WrNnMmh+0KOjSWbKBh+nGIKAqyiZNk+yR2w+T7ILr9bjJRwUGc47SKC/wvrrcdLNVaJAt5KqFe0o28r1dJev5tQ5zmJn95uZvJBt1nQ6NoOa4Bg2C+alNF9VyJbWn2eTAb7FHJj9/yvn676zo3GoUhKyR10c3ws7v10KylT10o995e+vw8JWt3kn4zE6SA1+5XLKikWMSPl7dxJ1fAAAARIPmFwAAANGg+QUAAEA0aH4BAAAQjSgG3spb6WDLcY2XBY/9y+LdJWv2eHqH21yB/mufd5luzrr6vKckO7lJ7mxYQWasO0qvtX1/MzYDlZid8+1Bkv1weUfJ8sZPl6zbWv251NEQ5Ip3lvUMpMtrvQ7gRwXbbC3Zme9+LNkhjRZKVmg6UFbo8tNT2M/Y53c6wF/0VGZe/zOJO78AAACIBs0vAAAAokHzCwAAgGjQ/AIAACAaUQy8VcWRzb6Q7KVBl0pWuNYnerylh6/Tx9vrbsm6FuiH359fo9teur14vmTTj7xXsrFLOwWqmb+JKpGrlp6tm9FOvOJ1yS5rOTVwdurvjUODG73uuliyDn/9KHC2DjGFdsNVRZ6r7iOgumae27la5094UjditbHQ9QPUDt9Sh+mPabw0cGS9mi8mIRdqXcrLar2OTOPOLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiEYUA2/lX0+RrOhp3XJiZjb1RB1GG3P9XWmt57V1rSQ7etivJet48zjJevZYqQ94pEbTxurAWxcG3nJaQacOkl13zUOSHdZolWTlgZGypWXFkh05Xq/Th7d/ULJuhfW1vvUS1Zpyz/v8TFvfaUOmSwDSa4FuXN193CmS7bzVPMnef3sHyRp+7xI97bo2OrX2l+OelOy4JoslG3DNaMlesX6SNX0yvZttsw2/IwAAACAaNL8AAACIBs0vAAAAokHzCwAAgGhEMfBmXj8c3u234Q9z7zb5QsnKByxL9DTLFzWVrPNzely918ZK1iGwqSi0iMWPnyzZ/y3eXrLTDnlXso9+nz1bZpC6/B7dgvlNox6VrEehblr7rlQH2QY8eqVk3e7+VrIt5ukmuIGP6M/M5AOG6bmHBAYub9X6amLb0PDHD5WsPdvBAFRD2TLtDbY8UrO5gXO3tY/TWssjd+4u2Z0PNJLs7R2ekuzd87rrAz5dO6/NmcKdXwAAAESD5hcAAADRoPkFAABANGh+AQAAEI04Bt6qoPV9gQ+h35fs3K3SW0pQfqstJNu5kQ7vjVu7bS1Ug0yYdn2TYB4abntznQ5h/vmvl0jW+QG97ksT1tPt9C8kO+7dwyUb1fsZyfYYfKlkWw1J/yBa+78x3FaXLChbK1mz75Jekbmj8XSGlJFM6YKFkjXROV+7Yuw+kr3S83nJ9jjvIsmC/VEdxZ1fAAAARIPmFwAAANGg+QUAAEA0aH4BAAAQDQbe6hjfTsfqDm+0WrJL399TsiL7rEZqQu16cI//JD72n5eeLtkWL9f80MKM17poqPMTdu7gkZK9OKRVDVSEuqRpng5vFjfTrGFtFJNQ/na6Jeu080ZV6zE7PTRTsvjG/mpXfsuWkvkNG4LHlq9ZU9PlpN1r7+0s2a0n60DwMRe+I9n79zWokZoygTu/AAAAiAbNLwAAAKJB8wsAAIBo0PwCAAAgGgy81THzDtINbyEFiwtruBJkSr75YJ4XeC9bf0lxTZcT1PlBHdR59IwOku3dcLpkL7cukqxs8ZL0FIaMaTpxE9vKDtGoiasv2Z6XjpVs0sPVrSp92j04X7LLW05LdO52D10YzLv8oP+fkT4FHdpL1uuFeZK99IIOkJuZdbwhezZHuvr6M/PdlbtI9vsBus0tpGO9xYFU/33VVdz5BQAAQDRofgEAABANml8AAABEg+YXAAAA0WDgrY4pbhkedkI8Hl2yVzDfue0Hks3+rR7X5aZekpV/+U216/opX1om2YqyRpJtV0/ffy86RgfeWt1fva10q07eQ7KmT35SrcdE1XR4cnb4Fy5Pdv4OjeZKNsm2Tr2gapj5dx2AerrdvwNH6hDS/St08LPbrTr4aWZWVso+t5q0Yrd2kv29zYuSXXPuh8Hzd2mtL7A9hq2sfmE/MfOEFpKVtCyX7MYDn5XsxCY6kJdnTjJ9NLO7bzxesuaWO6+Z3PkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRYOANqGNef/MX4V84Qwfexu8zXLL5L+jWt1sW9Zfs1fd3rnpxlUYce5tkPQrzJfuiWN9/b/nYV5KFBjKq4vg/vi7ZqCebVfNRURV+zZpgfvuybpJd2lIHwH7V9DvJ/vrwAMl6/GutZOXjJycpMWj1CbtL9sVpt0rWMLCVLjTc9uJxOrBa9kOyTXBIr8bz1kn2f4u3l+yPrScEz59y7N2S5R0bGijTQfXw4FmygfbqnLuoTH8+9n7hCsmKnv1Cslwat+fOLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKLBtz3kgHyn72FaTsxAIagV3W6bEcw/PalQst3rl0jWvqChZLcEViPfcpJmSeWZ1lIe+M6GV1ftqMet1Wnk6rp/0t6SdbSv0/482LSy5SuC+VsDdbreXtIo9A0Q0/oPk+yR3XTl8T+e1FWtIace+7ZmzW+RrKHTVd0hdz56lGTtJ+nKWWTIJ+Mleu9yXV198B/0G0nMzP7b8ynJmgS+9SP07QwhSb/F4bFVW0l2fJP5kvV+bbBknUboc3R/+VPJcumbHUK48wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKLBwFsOKPM6SNRy0uoMVILaUPb9omD+90OPk2zK4C0lG9T/Lcku2+Kb6hf2E+d8t79kY0fpYFOX4bqy1mxuWmsxM+t4AsNt2ap0tl4Dj99+iB54aSAKDMGd3nShZucNSam2Cjrc9uDKtpI9d/x+krWfpINEyG4Fb43TUF8yzczsyCP0opz/qw2SjdlX1yAfP+VkyRa/1F4yF9jt3vYxXdf90E46XFn09md6MsyMO78AAACICM0vAAAAokHzCwAAgGjQ/AIAACAaDLzlgNCGN8SnbKpufut2mWZvW+NA1jfN1ayUpKPpZqvSND8rckOr+z+W7PUHW0v2Zuc+kk2+SLdf7bObDnR+MKZXolp6Dl0mWfnUWZL5kimJHg+5o8HIMZJ1GanHnWx7SVZgOui5dSALKQtkBW8vTXQuKtA1AQAAIBo0vwAAAIgGzS8AAACiQfMLAACAaDDwlgNmlOg2t/zlayULfUgeAOoCX6Kbs8qmzZSs+6WafR94vO72SaLn5XUTyD3c+QUAAEA0aH4BAAAQDZpfAAAARIPmFwAAANFg4K2O6fxH3Xw0+I/7BI7UzV4AAACx484vAAAAokHzCwAAgGjQ/AIAACAaNL8AAACIBs0vAAAAokHzCwAAgGjQ/AIAACAaNL8AAACIBs0vAAAAouG895muAQAAAKgV3PkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANGh+AQAAEA2aXwAAAESD5hcAAADRoPkFAABANGh+a4lzzjvn1jjn/prw+HOcc6srz+tW0/UBm8K1i7qKaxd1UQrX7Q2Vx3vnXEFN15cLnPc+0zVEwTnnzay79376T7KhZrafmXU3s1977x9Mch5Qmza+Bp1zRWb2TzPby8zyzWysmV3ivZ/yc+cBtS1w7e5rZq9udFhjMzvee//cps4DatMm+oU+ZjbczLYzs0lmdo73/suf/HpnM5tlZoXe+9JaLbgO4s5vZn1lZoPN7PNMFwJUQQsze9HMephZGzMbY2YvZLQiIAHv/fve+yY//mVmA81stZm9luHSgE1yztWzitfYR82spZk9ZGYvVOZIAc1vBnnv7/Lev2Vm6zNdC5CU936M9364936p977EzG41sx7OuVaZrg2oojPN7Fnv/ZpMFwL8jH5mVmBmt3nvi733d5iZM7MDMlpVHUbzC6C6fmlmC733SzJdCJCUc66RmR1vFXfRgGzW28zG+//9nOr4yhwpoPkFkDLnXHszu8vMLs90LUAVHWdmi83s3UwXAmxGEzNbsVG2wsyaZqCWnEDzCyAlzrktzex1M7vbe/9EpusBquhMM3vYM/WN7LfazJptlDUzs1UZqCUn0PwCqDLnXEuraHxf9N4n+joeIFs45zpYxecoH85wKUASE81sR+ec+0m2Y2WOFND8ZpBzrp5zroFVfHC90DnXwDnHfxNkNedcMzMbZWYfeu+vznQ9QApON7OPvPczMl0IkMBoMyszs0ucc/WdcxdV5m9nrqS6jUYrs143s3VW8X2pQyv/9y8zWhGweceYWV8zO7tyIcCPf3XMdGFAQmcYg26oI7z3G8zsaKu4bpeb2a/N7OjKHCmg+a09xWY2zjl344+B976f995t9NdoMzPn3NnOueWV55VnpmTAzDa6dr33D1Veq41/+p2p3vvvzLh2kVXkddfMzHvf03s/fOODuXaRJUL9whfe+1289w2997/w3n/x46855663ir0BxWbGZ9gTYMMbAAAAosGdXwAAAESD5hcAAADRKKjNJzso7wQ+Y4Eqe6P8Gbf5o2oW1y5Skelrl+sWqcj0dWvGtYvUJL12ufMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiQfMLAACAaND8AgAAIBo0vwAAAIhGQaYLiNG0h34h2ZQD75fsgIsGS9ZoxKc1UhMAAMh++b17SDb72FaS7TpggmQPd3pPshJflnIt/S+8IJg3fH5Myo9ZG7jzCwAAgGjQ/AIAACAaNL8AAACIBs0vAAAAosHAWyZ4J1G5lUs2r7+e2n1ETRSEXFWwbSfJ5hzTTrJVRaWS9SiaJ9nIHi9KVvTS+ZK1H6Xvq5t9sVAyv3qtZGU//CCZK9CXqvmX7CZZaUOJzMys47/G6XMXF4cPBoAssfKUPSQ7/OrRko1o9XWixyvx+toc6j+Suue224P5lVPOkKxs0rSUnyfduPMLAACAaND8AgAAIBo0vwAAAIgGzS8AAACiwcBbFuu63XzJXP36kjG4AzOzhZftJdlnV94pWXWGG0JnTh14rx43MNlzPLVqG8n+89tjJJu/r75UfX1meNAi5IjR50nmPvwy8fkAkE55DRpINuPPO0s28fQhklXnNTzdigrrBfNJl7bUY3U2OmO48wsAAIBo0PwCAAAgGjS/AAAAiAbNLwAAAKLBwFsWe6Xn85Id1eQgycoYeItKfrdtg/lDl94aSNP7Iz5i9VaSHddkccqPd1LTBZoNu1uyvMD79NDIxxfF4ffz+SvWJzofNef7S3Qg08xs5a763ybbFdbXjYgT9nkg0bkD2+2S7nKQ7ZxudQ0Nt319+h2Bk2v+Hu6vlFEAAAu9SURBVGWvpy9OdNw3J+oA9abctP8zkj2w20A9cEyyzXTpxp1fAAAARIPmFwAAANGg+QUAAEA0aH4BAAAQDZpfAAAARINvewDqmPkDdCWwmdl29ZK9lz3g65Mka3xjs0TnFi5YLtnwti0kK26lKy8H36zTv8c0WZToeUMmbPCSXXnF4OCxjSZ8mvLzID3W7LE2mE/a7/5E54e/8SO939lRnecIHfXoyg7VrAh1Tfm++i0OMwfpcd8cEPpmh2SeXb21ZH/8QNfCd3hRr+eGL4yRrJt9Ipnbubc+8YkJC7Twa/sdXRpL1lTLqRXc+QUAAEA0aH4BAAAQDZpfAAAARIPmFwAAANFg4A2oY/Y5fVziYxeUrZPs+6/bSJZ/WLLHa/OZDrJ9v2u+ZPscqCsrqzPcFvLSyj6SNRrBYFu26j54VjA/tqkO6sw6q6NkxS11pMzpzGO1lLfeINmkA+9LdG7PV3TYcrvfTw8cuayqZSFbBdYWh4fbhqb8FEdMOVKy8uu2lKzow89Sfo4YcecXAAAA0aD5BQAAQDRofgEAABANml8AAABEg4E3oI55+bOdgvnNR7wvWceCJpJNOmVI6k9+tkaFTgfeSnxZ4GR9r704MJC37zO/k2z0Cf+S7JrWOlTX78QLA89r1uRp3WCE2lW2fEX4FwJ5hxvn1nA1YatP3EPDAzWaXlIq2Xb/XCpZ2TKG23JBXoMGwXzGn3WbW3U2t31aXCiZP2CeZM40Q9Vw5xcAAADRoPkFAABANGh+AQAAEA2aXwAAAESDgTegjim6YEww/0WrcyT7eu8HJSs33ZRVHSWBLVsvrmkp2e2z+kuWd3trybq+osNp+za+XLLJR9wl2fyDQoN2ZkVPB2PgfywYqBveQm6YO1Cysqkz0l0OsoTfrmsw//r01IfbtnvrN5J1HaqvzXn2ZcrPgU3jzi8AAACiQfMLAACAaND8AgAAIBo0vwAAAIgGA29Ajuhygw7r9Osd3nhW01p8tlCyhjNnBY4MZanboWhOMC9O67MgV03rP0yy8sA9onFjukvWzZbUSE3IvHn9mwfzvIT3D0es2UKy7kNK9MAxurUym4T+/4Y2fJqFB6G9S3dFqePOLwAAAKJB8wsAAIBo0PwCAAAgGjS/AAAAiAYDb0COKJs4RbImEzNQiJmVpvnxehTNS3Tc11M7BPMi0wE8YGPlplM6oY2ILjDMg9xQ0KG9ZIee8nHw2KTbMq96+yTJisaEN3X+v/buNlbrso4D+H2AwwniBB0YBAgIyqOKMKCS0HI8rJaj2VpkbtpGD7q0XAXLNisX8QJTA1sjKEgLl2wNmxPBdIxsCA2ELB5GoCAkIA/ytMAOh9OL3vW7WH/OOTf3fc71+bz87n/d/+so3Hx3bb9zVYuDD8Us9fOmBttKpVLp7n3TQvbB53eELH0nZ/k5+QUAIBvKLwAA2VB+AQDIhvILAEA2DLxVQmJa4nJuToGOrHHahJCtHbkkZK++VxuykT//V/IzzSfxv8595sOJdEuhtU0N8XauN54eF7IJQ94K2QP9/xg/r5S++uory+4L2aB5G4pskRY6cXMceJvXb1Xh9dP//vmQjZ67K2SVGvRK2ffM2JAtG/frVn3m3sWjQtbrdHpwsBKc/AIAkA3lFwCAbCi/AABkQ/kFACAbBt4qoTkONxS9OWXn/GtCNuJrJ9pkW3CldaqvD9n8JXG4LTX8+aezcaCieWuFrrSjxTr36xuyM5OHhuxcQzyr6fTZYy1+75PX/TSR1hVau2vG4ha/d/b+6SHbsmZM8tmrH9sWsmJ3itFSx2emh2aLOnCwd8hGnH6zVZ9ZbnPHvhiyiXXFRvJmv3VrMu+9Zk/IqmnIz8kvAADZUH4BAMiG8gsAQDaUXwAAsmHgrb3patyB9qlz74aQnX26Z8jG1xUb/ly2/uMhG17a1LLNUXaNMyYm8/qH9oVs1bCfhSx1C2ZqULi4eENgUamhtaPfGlxs8cbXQzS4lL61zbf9lfe9cWtClvqzdykjZm9uy+20udMvxKH5uz6Qutmw2M+8Y9l1ybz30eq5zS3FyS8AANlQfgEAyIbyCwBANpRfAACyYeANuCIOzI43sm2+fmGhtfOOjQ3Z6MePhOzC5W+LK2T/p9L/3KwdtjZkK84MDNnJpu4h+8PbN4bsnXVxbcqi2b8I2dRu8Q6qSa/dEbKG23YnPvFkofdS3Zqa23qw8sro3CsOD+9ZPCRk28cuD1nRn2/MyvtDdu3S6h5suxQnvwAAZEP5BQAgG8ovAADZUH4BAMiGgTegVTrV14fsyIoBIfv9jY8kVncNyaJ342Dc2gU3h6znGxuLbZCq0GtnTTIfsfqekI2eEwfKmk6eClnX0v6QXZXIUv76xTgMdMv7/lFoLVRS47QJIev3o/hnd9XgXyVWFzvzfOlc/F4fufREyOKIaPvg5BcAgGwovwAAZEP5BQAgG8ovAADZMPAG7czhByYn867TjoXs0TErQ3YxcYNRUT/e9+mQPTz02ZCNr0vdGBSH21LWzZoYsp7bDbe1d32WpG+C6rMkZtU0RFP724ZKb4F25PiXbwpZ71+2/Ba03cvjcNuQgcdDtnTwyy1+R8r9L9wdsuE7NrXpOyrJyS8AANlQfgEAyIbyCwBANpRfAACyYeANqtiZWR8N2eY5TxReX1vTOWSNzY0t3s/qUXG4Lf2OuPbUxfMhm/qTOSH70PYNLdscXELnfn1DNqC22E1wXc6nhjfpqBa8PiNkd01ZXnj9mNnbQ7a5fxxS/uoXVofs6732hqy2ZlvIGptTI6HFzjJT39cjnrwvZMMfbPmQXnvg5BcAgGwovwAAZEP5BQAgG8ovAADZMPBWZjVd4n/iuvf/uwI7oT06NP1CyC6Wig/gpAbPLmd9W77j+4enhmzgi0dDVk23e9ExnJk8NGS393g+8aTzoNxdtaQ2ZK9OikNipVKp9JG6ODycvGntnmK3r6W+mVvzHb7o3VEhW/pcHOgb9sPXQpZ4bYfibzoAANlQfgEAyIbyCwBANpRfAACyofwCAJANv+2hzDoNHRyybZOXFVqbug52wGr/yzqqzr0bQnbHhL9UYCfl8fiAV0K27rkeIXtiyidCduHwkXJsiYx1Spz9pL5zu5z1+0dy0uXlLSH7zsP3Jp99Zf6icm8n6eCF90K24Mj0kB340qCQDd0Rry3u6L/ZIcXJLwAA2VB+AQDIhvILAEA2lF8AALJheqrcTpwM0Q1PfSNkE2/ZFbKDjwwPWY9nN7XNvqg6jWOGhOwHfddWYCf/9ckdnwvZkfUD44M1MfrunStDNqv+UMhu7XY2ZE/UdS22QWiF1BWxT526IWS1L8UBKPLSZ83eZD5+0DdDtvXeheXeTun2hXND1v+xDYknd5d9L+2Vk18AALKh/AIAkA3lFwCAbCi/AABkw8BbmTUdPxGyoQ/GG1aOJ9Z2K3Wc2734/2oPxeHIKVvvDNmfx68o/JmHms6FbPpv5oTs2iUHQ1b3dhxQG9S4v9B7f7d4fMie6X5TyE5PGBCy+tOGNKiMpTs/FrLBpb9VYCdUk6Yj7yTzQfNiPnPepHJvp9S/lBpu43I4+QUAIBvKLwAA2VB+AQDIhvILAEA2DLxBlWja82bIGm6Lz80stW6g4upSHLi80KpPjJqOHi30XPf9B+LaNt4L/HNased6rO5R3o0AVcHJLwAA2VB+AQDIhvILAEA2lF8AALJh4A2ADq3T+ZqQPXr8+pA1LI/DoEDH4+QXAIBsKL8AAGRD+QUAIBvKLwAA2TDwBkCHds23N4ZsfalbBXYCVAMnvwAAZEP5BQAgG8ovAADZUH4BAMhGTXNzc6X3AAAAV4STXwAAsqH8AgCQDeUXAIBsKL8AAGRD+QUAIBvKLwAA2VB+AQDIhvILAEA2lF8AALKh/AIAkA3lFwCAbCi/AABkQ/kFACAbyi8AANlQfgEAyIbyCwBANpRfAACyofwCAJAN5RcAgGwovwAAZEP5BQAgG8ovAADZUH4BAMjGfwDMUMjfgPRICQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x864 with 16 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "plt.figure(figsize=(12,12))\n",
    "\n",
    "for idx in range(16):\n",
    "    plt.subplot(4,4,idx+1)\n",
    "    plt.axis('off')\n",
    "    plt.title('[{}]'.format(mnist.train.labels[idx]))\n",
    "    plt.imshow(mnist.train.images[idx].reshape((28,28)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(\"float\", [None, 784])\n",
    "y = tf.placeholder(\"int64\", [None])\n",
    "learning_rate = tf.placeholder(\"float\")\n",
    "\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "  return tf.truncated_normal(shape, stddev=0.1)\n",
    "\n",
    "L1_units_count = 200\n",
    "\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count]))\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))\n",
    "logits_1 = tf.matmul(x, W_1) + b_1\n",
    "output_1 = tf.nn.relu(logits_1)\n",
    "\n",
    "L2_units_count = 10 \n",
    "W_2 = tf.Variable(initialize([L1_units_count, L2_units_count]))\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "\n",
    "logits = logits_2\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sparse_softmax_cross_entropy_with_logits ,lable可以不用手动做one-hot\n",
    "cross_entropy_loss = tf.reduce_mean(\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(\n",
    "    learning_rate=learning_rate).minimize(cross_entropy_loss)\n",
    "\n",
    "pred = tf.nn.softmax(logits)\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), y)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "steps:100,train_loss: 0.619024,train_acc: 0.878545 ,val_acc: 0.8894\n",
      "steps:200,train_loss: 0.287759,train_acc: 0.912364 ,val_acc: 0.9194\n",
      "steps:300,train_loss: 0.319781,train_acc: 0.923382 ,val_acc: 0.9242\n",
      "steps:400,train_loss: 0.12873,train_acc: 0.932273 ,val_acc: 0.9346\n",
      "steps:500,train_loss: 0.484202,train_acc: 0.9462 ,val_acc: 0.9436\n",
      "steps:600,train_loss: 0.170845,train_acc: 0.950182 ,val_acc: 0.9522\n",
      "steps:700,train_loss: 0.106595,train_acc: 0.948709 ,val_acc: 0.9472\n",
      "steps:800,train_loss: 0.16412,train_acc: 0.942255 ,val_acc: 0.9454\n",
      "steps:900,train_loss: 0.0709884,train_acc: 0.945745 ,val_acc: 0.9456\n",
      "steps:1000,train_loss: 0.411162,train_acc: 0.958382 ,val_acc: 0.9592\n",
      "steps:1100,train_loss: 0.114122,train_acc: 0.965073 ,val_acc: 0.9632\n",
      "steps:1200,train_loss: 0.319377,train_acc: 0.963745 ,val_acc: 0.9582\n",
      "steps:1300,train_loss: 0.2677,train_acc: 0.968745 ,val_acc: 0.9678\n",
      "steps:1400,train_loss: 0.0403949,train_acc: 0.970891 ,val_acc: 0.9684\n",
      "steps:1500,train_loss: 0.0945635,train_acc: 0.967218 ,val_acc: 0.9656\n",
      "steps:1600,train_loss: 0.18461,train_acc: 0.972655 ,val_acc: 0.9702\n",
      "steps:1700,train_loss: 0.0265723,train_acc: 0.973109 ,val_acc: 0.9684\n",
      "steps:1800,train_loss: 0.0286024,train_acc: 0.972236 ,val_acc: 0.9682\n",
      "steps:1900,train_loss: 0.0954772,train_acc: 0.971891 ,val_acc: 0.967\n",
      "steps:2000,train_loss: 0.0714216,train_acc: 0.977145 ,val_acc: 0.9724\n",
      "steps:2100,train_loss: 0.268506,train_acc: 0.979582 ,val_acc: 0.973\n",
      "steps:2200,train_loss: 0.0402717,train_acc: 0.980782 ,val_acc: 0.9752\n",
      "steps:2300,train_loss: 0.039879,train_acc: 0.9812 ,val_acc: 0.9748\n",
      "steps:2400,train_loss: 0.0224946,train_acc: 0.981709 ,val_acc: 0.9766\n",
      "steps:2500,train_loss: 0.0181764,train_acc: 0.981982 ,val_acc: 0.977\n",
      "steps:2600,train_loss: 0.198204,train_acc: 0.982073 ,val_acc: 0.9764\n",
      "steps:2700,train_loss: 0.0373511,train_acc: 0.982909 ,val_acc: 0.9766\n",
      "steps:2800,train_loss: 0.0307868,train_acc: 0.9832 ,val_acc: 0.9776\n",
      "steps:2900,train_loss: 0.0152367,train_acc: 0.983727 ,val_acc: 0.978\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.9752\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "trainig_step = 3000\n",
    "saver = tf.train.Saver()\n",
    "lr=0.5\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    #定义验证集与测试集\n",
    "    train_data={x: mnist.train.images, y:mnist.train.labels}\n",
    "    validate_data = {\n",
    "        x: mnist.validation.images,\n",
    "        y: mnist.validation.labels,\n",
    "    }\n",
    "    test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "\n",
    "    for i in range(trainig_step):\n",
    "        if i > 1000:\n",
    "            lr = 0.3\n",
    "        if i > 2000:\n",
    "            lr = 0.1\n",
    "        xs, ys = mnist.train.next_batch(batch_size)\n",
    "        _, loss = sess.run(\n",
    "            [optimizer, cross_entropy_loss],\n",
    "            feed_dict={\n",
    "                x: xs,\n",
    "                y: ys,\n",
    "                learning_rate:lr\n",
    "            })\n",
    "\n",
    "        #每100次训练打印一次损失值与验证准确率\n",
    "        if i > 0 and i % 100 == 0:\n",
    "            validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "            train_acc=sess.run(accuracy,feed_dict=train_data)\n",
    "            print(\n",
    "                \"steps:%d,train_loss: %g,train_acc: %g ,val_acc: %g\"\n",
    "                % (i, loss, train_acc,validate_accuracy))\n",
    "            \n",
    "\n",
    "    print(\"the training is finish!\")\n",
    "    #最终的测试准确率\n",
    "    acc = sess.run(accuracy, feed_dict=test_data)\n",
    "    print(\"the test accuarcy is:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增加隐层及每层神经元数量的设定：相比单一的线性模型，此次模型训练增加了一个隐层L2层，L2层的输入为L1层经过ReLu激活的输出，L1层的units_count为100，L2层的units_count为10对应10个数字的s输出，模型增加了非线性激活函数，增加了模型的拟合能力，提高了模型的准确率。\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "参数初始化方法：1、截断高斯分布 tf.truncated_normal（本文使用）\n",
    "2、xavier初始化(适用于激活函数是sigmoid和tanh) tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32) \n",
    "3、MSRA初始化(适用于激活函数relu)tf.contrib.layers.variance_scaling_initializer(factor=2.0, mode='FAN_IN', uniform=False, seed=None, dtype=tf.float32)  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "神经元个数分别设置为50，100，150，200，模型准确率在神经元个数为150和200时，准确率达到97%以上，不过神经元数量越多，参数增多，计算量增加。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>添加双隐层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "regularizer=tf.contrib.layers.l2_regularizer(0.01)\n",
    "x = tf.placeholder(\"float\", [None, 784])\n",
    "y = tf.placeholder(\"int64\", [None])\n",
    "learning_rate = tf.placeholder(\"float\")\n",
    "\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "  return tf.truncated_normal(shape, stddev=0.1)\n",
    "\n",
    "L1_units_count = 200\n",
    "\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count]))\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))\n",
    "logits_1 = tf.matmul(x, W_1) + b_1\n",
    "output_1 = tf.nn.relu(logits_1)\n",
    "\n",
    "L2_units_count = 100\n",
    "W_2 = tf.Variable(initialize([L1_units_count, L2_units_count]))\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "output_2 = tf.nn.relu(logits_2)\n",
    "\n",
    "L3_units_count = 10 \n",
    "W_3 = tf.Variable(initialize([L2_units_count, L3_units_count]))\n",
    "b_3 = tf.Variable(initialize([L3_units_count]))\n",
    "logits_3 = tf.matmul(output_2, W_3) + b_3  \n",
    "\n",
    "logits = logits_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sparse_softmax_cross_entropy_with_logits ,lable可以不用手动做one-hot\n",
    "cross_entropy_loss = tf.reduce_mean(\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(\n",
    "    learning_rate=learning_rate).minimize(cross_entropy_loss)\n",
    "\n",
    "pred = tf.nn.softmax(logits)\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), y)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "steps:100,train_loss: 0.622751,train_acc: 0.8132 ,val_acc: 0.8184\n",
      "steps:200,train_loss: 0.17677,train_acc: 0.912182 ,val_acc: 0.9204\n",
      "steps:300,train_loss: 0.108651,train_acc: 0.911964 ,val_acc: 0.9138\n",
      "steps:400,train_loss: 0.098534,train_acc: 0.927036 ,val_acc: 0.9296\n",
      "steps:500,train_loss: 0.167848,train_acc: 0.934236 ,val_acc: 0.94\n",
      "steps:600,train_loss: 0.0843422,train_acc: 0.949 ,val_acc: 0.9532\n",
      "steps:700,train_loss: 0.07676,train_acc: 0.936855 ,val_acc: 0.9404\n",
      "steps:800,train_loss: 0.171991,train_acc: 0.948327 ,val_acc: 0.9502\n",
      "steps:900,train_loss: 0.186291,train_acc: 0.950764 ,val_acc: 0.9524\n",
      "steps:1000,train_loss: 0.202575,train_acc: 0.956509 ,val_acc: 0.957\n",
      "steps:1100,train_loss: 0.0427341,train_acc: 0.964727 ,val_acc: 0.9664\n",
      "steps:1200,train_loss: 0.12782,train_acc: 0.969182 ,val_acc: 0.971\n",
      "steps:1300,train_loss: 0.286455,train_acc: 0.970618 ,val_acc: 0.9716\n",
      "steps:1400,train_loss: 0.0405475,train_acc: 0.972182 ,val_acc: 0.9728\n",
      "steps:1500,train_loss: 0.0417965,train_acc: 0.972818 ,val_acc: 0.9728\n",
      "steps:1600,train_loss: 0.0721801,train_acc: 0.973273 ,val_acc: 0.9716\n",
      "steps:1700,train_loss: 0.069276,train_acc: 0.975309 ,val_acc: 0.9752\n",
      "steps:1800,train_loss: 0.0767961,train_acc: 0.9752 ,val_acc: 0.9742\n",
      "steps:1900,train_loss: 0.0158016,train_acc: 0.975891 ,val_acc: 0.9736\n",
      "steps:2000,train_loss: 0.181426,train_acc: 0.976509 ,val_acc: 0.9748\n",
      "steps:2100,train_loss: 0.14249,train_acc: 0.977945 ,val_acc: 0.976\n",
      "steps:2200,train_loss: 0.110714,train_acc: 0.978091 ,val_acc: 0.9744\n",
      "steps:2300,train_loss: 0.00970852,train_acc: 0.978364 ,val_acc: 0.9764\n",
      "steps:2400,train_loss: 0.0535513,train_acc: 0.979091 ,val_acc: 0.9758\n",
      "steps:2500,train_loss: 0.0879894,train_acc: 0.978655 ,val_acc: 0.9768\n",
      "steps:2600,train_loss: 0.0406467,train_acc: 0.979073 ,val_acc: 0.9766\n",
      "steps:2700,train_loss: 0.0580733,train_acc: 0.980055 ,val_acc: 0.9768\n",
      "steps:2800,train_loss: 0.020548,train_acc: 0.980273 ,val_acc: 0.9772\n",
      "steps:2900,train_loss: 0.0300531,train_acc: 0.979982 ,val_acc: 0.9764\n",
      "steps:3000,train_loss: 0.134796,train_acc: 0.980145 ,val_acc: 0.977\n",
      "steps:3100,train_loss: 0.0705333,train_acc: 0.980655 ,val_acc: 0.9778\n",
      "steps:3200,train_loss: 0.249861,train_acc: 0.980909 ,val_acc: 0.978\n",
      "steps:3300,train_loss: 0.276457,train_acc: 0.980709 ,val_acc: 0.9774\n",
      "steps:3400,train_loss: 0.18816,train_acc: 0.980727 ,val_acc: 0.9772\n",
      "steps:3500,train_loss: 0.049303,train_acc: 0.980873 ,val_acc: 0.9774\n",
      "steps:3600,train_loss: 0.124791,train_acc: 0.981109 ,val_acc: 0.9778\n",
      "steps:3700,train_loss: 0.0529426,train_acc: 0.981164 ,val_acc: 0.9778\n",
      "steps:3800,train_loss: 0.00680914,train_acc: 0.981364 ,val_acc: 0.9782\n",
      "steps:3900,train_loss: 0.151492,train_acc: 0.981418 ,val_acc: 0.978\n",
      "steps:4000,train_loss: 0.0145944,train_acc: 0.981673 ,val_acc: 0.978\n",
      "steps:4100,train_loss: 0.0235418,train_acc: 0.981564 ,val_acc: 0.978\n",
      "steps:4200,train_loss: 0.111718,train_acc: 0.981655 ,val_acc: 0.9782\n",
      "steps:4300,train_loss: 0.245625,train_acc: 0.9816 ,val_acc: 0.9776\n",
      "steps:4400,train_loss: 0.0553862,train_acc: 0.981818 ,val_acc: 0.978\n",
      "steps:4500,train_loss: 0.0196666,train_acc: 0.981927 ,val_acc: 0.9776\n",
      "steps:4600,train_loss: 0.0174313,train_acc: 0.982 ,val_acc: 0.9778\n",
      "steps:4700,train_loss: 0.0319542,train_acc: 0.981964 ,val_acc: 0.9778\n",
      "steps:4800,train_loss: 0.0561741,train_acc: 0.982055 ,val_acc: 0.9772\n",
      "steps:4900,train_loss: 0.159552,train_acc: 0.982018 ,val_acc: 0.9776\n",
      "steps:5000,train_loss: 0.0410719,train_acc: 0.982109 ,val_acc: 0.9776\n",
      "steps:5100,train_loss: 0.00633544,train_acc: 0.982018 ,val_acc: 0.978\n",
      "steps:5200,train_loss: 0.0574464,train_acc: 0.982036 ,val_acc: 0.9778\n",
      "steps:5300,train_loss: 0.234458,train_acc: 0.982055 ,val_acc: 0.978\n",
      "steps:5400,train_loss: 0.116816,train_acc: 0.982036 ,val_acc: 0.978\n",
      "steps:5500,train_loss: 0.0357343,train_acc: 0.982127 ,val_acc: 0.978\n",
      "steps:5600,train_loss: 0.0815959,train_acc: 0.982145 ,val_acc: 0.978\n",
      "steps:5700,train_loss: 0.0214846,train_acc: 0.982073 ,val_acc: 0.9778\n",
      "steps:5800,train_loss: 0.110786,train_acc: 0.982091 ,val_acc: 0.9778\n",
      "steps:5900,train_loss: 0.159497,train_acc: 0.982073 ,val_acc: 0.978\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.9733\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "trainig_step = 6000\n",
    "saver = tf.train.Saver()\n",
    "lr=0.5\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    #定义验证集与测试集\n",
    "    train_data={x: mnist.train.images, y:mnist.train.labels}\n",
    "    validate_data = {\n",
    "        x: mnist.validation.images,\n",
    "        y: mnist.validation.labels,\n",
    "    }\n",
    "    test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "\n",
    "    for i in range(trainig_step):\n",
    "        if i > 1000:\n",
    "            lr = 0.1\n",
    "        if i > 2000:\n",
    "            lr = 0.05\n",
    "        if i>3000:\n",
    "            lr =0.01\n",
    "        if i > 4000:\n",
    "            lr = 0.005\n",
    "        if i>5000:\n",
    "            lr =0.001\n",
    "        xs, ys = mnist.train.next_batch(batch_size)\n",
    "        _, loss = sess.run(\n",
    "            [optimizer, cross_entropy_loss],\n",
    "            feed_dict={\n",
    "                x: xs,\n",
    "                y: ys,\n",
    "                learning_rate:lr\n",
    "            })\n",
    "\n",
    "        #每100次训练打印一次损失值与验证准确率\n",
    "        if i > 0 and i % 100 == 0:\n",
    "            validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "            train_acc=sess.run(accuracy,feed_dict=train_data)\n",
    "            print(\n",
    "                \"steps:%d,train_loss: %g,train_acc: %g ,val_acc: %g\"\n",
    "                % (i, loss, train_acc,validate_accuracy))\n",
    "            \n",
    "\n",
    "    print(\"the training is finish!\")\n",
    "    #最终的测试准确率\n",
    "    acc = sess.run(accuracy, feed_dict=test_data)\n",
    "    print(\"the test accuarcy is:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练采用双隐层结构，激活函数为ReLu,没有出现明显过拟合现象"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L1正则化和L2正则化都能限制模型复杂度，改善模型过拟合的效果，L1正则会将高此项参数权重减小甚至趋向0,减少参数数量，产生稀疏权值矩阵，即产生一个稀疏模型，多用于特征选择。\n",
    "L2正则可以使各项参数权值减小，减小模型复杂度。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
